{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T00:17:23.837595Z",
     "start_time": "2017-10-27T00:17:23.816518Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import itertools\n",
    "import bcolz\n",
    "import pickle\n",
    "sys.path.append('../../lib')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import random\n",
    "import smart_open\n",
    "import h5py\n",
    "import csv\n",
    "import json\n",
    "import functools\n",
    "import time\n",
    "import string\n",
    "\n",
    "import datetime as dt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import global_utils\n",
    "\n",
    "random_state_number = 967898"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:11:59.986207Z",
     "start_time": "2017-10-26T21:11:59.524390Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0', '/gpu:1']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:00.074968Z",
     "start_time": "2017-10-26T21:11:59.987453Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: TkAgg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['random']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab\n",
    "%matplotlib inline\n",
    "%load_ext line_profiler\n",
    "%load_ext memory_profiler\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:00.079288Z",
     "start_time": "2017-10-26T21:12:00.076300Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_columns = 999\n",
    "color = sns.color_palette()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:17.191979Z",
     "start_time": "2017-10-26T21:12:00.080389Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')\n",
    "train_df = store['train_df']\n",
    "test_df = store['test_df']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:17.503266Z",
     "start_time": "2017-10-26T21:12:17.193184Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Class</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[fam58a]</td>\n",
       "      <td>[truncating, mutations]</td>\n",
       "      <td>1</td>\n",
       "      <td>[[cyclin-dependent, kinases, , cdks, , regulat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[w802*]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[q249e]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[n454d]</td>\n",
       "      <td>3</td>\n",
       "      <td>[[recent, evidence, has, demonstrated, that, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[l399v]</td>\n",
       "      <td>4</td>\n",
       "      <td>[[oncogenic, mutations, in, the, monomeric, ca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID      Gene                Variation  Class  \\\n",
       "0   0  [fam58a]  [truncating, mutations]      1   \n",
       "1   1     [cbl]                  [w802*]      2   \n",
       "2   2     [cbl]                  [q249e]      2   \n",
       "3   3     [cbl]                  [n454d]      3   \n",
       "4   4     [cbl]                  [l399v]      4   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[cyclin-dependent, kinases, , cdks, , regulat...  \n",
       "1  [[abstract, background, non-small, cell, lung,...  \n",
       "2  [[abstract, background, non-small, cell, lung,...  \n",
       "3  [[recent, evidence, has, demonstrated, that, a...  \n",
       "4  [[oncogenic, mutations, in, the, monomeric, ca...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[acsl4]</td>\n",
       "      <td>[r570s]</td>\n",
       "      <td>[[2, this, mutation, resulted, in, a, myelopro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[naglu]</td>\n",
       "      <td>[p521l]</td>\n",
       "      <td>[[abstract, the, large, tumor, suppressor, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[pah]</td>\n",
       "      <td>[l333f]</td>\n",
       "      <td>[[vascular, endothelial, growth, factor, recep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[ing1]</td>\n",
       "      <td>[a148d]</td>\n",
       "      <td>[[inflammatory, myofibroblastic, tumor, , imt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[tmem216]</td>\n",
       "      <td>[g77a]</td>\n",
       "      <td>[[abstract, retinoblastoma, is, a, pediatric, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       Gene Variation                                          Sentences\n",
       "0   0    [acsl4]   [r570s]  [[2, this, mutation, resulted, in, a, myelopro...\n",
       "1   1    [naglu]   [p521l]  [[abstract, the, large, tumor, suppressor, 1, ...\n",
       "2   2      [pah]   [l333f]  [[vascular, endothelial, growth, factor, recep...\n",
       "3   3     [ing1]   [a148d]  [[inflammatory, myofibroblastic, tumor, , imt,...\n",
       "4   4  [tmem216]    [g77a]  [[abstract, retinoblastoma, is, a, pediatric, ..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head())\n",
    "display(test_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:17.603365Z",
     "start_time": "2017-10-26T21:12:17.504374Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352220 352220\n"
     ]
    }
   ],
   "source": [
    "corpus_vocab_list, corpus_vocab_wordidx = None, None\n",
    "with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:\n",
    "    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)\n",
    "print(len(corpus_vocab_list), len(corpus_wordidx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-14T08:20:17.449244Z",
     "start_time": "2017-08-14T08:20:15.593136Z"
    },
    "collapsed": true
   },
   "source": [
    "# Data Prep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To control the vocabulary pass in updated corpus_wordidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:17.643977Z",
     "start_time": "2017-10-26T21:12:17.604523Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5)\n",
      "(333, 5)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train_df, x_val_df = train_test_split(train_df,\n",
    "                                         test_size=0.10, random_state=random_state_number,\n",
    "                                         stratify=train_df.Class)\n",
    "\n",
    "print(x_train_df.shape)\n",
    "print(x_val_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:24.734305Z",
     "start_time": "2017-10-26T21:12:17.645066Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.contrib.keras.python.keras.utils import np_utils\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:12:24.737965Z",
     "start_time": "2017-10-26T21:12:24.735657Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size=len(corpus_vocab_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:sent_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:14:20.659301Z",
     "start_time": "2017-10-26T00:14:20.653375Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"sentences\",\n",
    "         \"divide_document\": \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:16:51.148007Z",
     "start_time": "2017-10-26T00:14:22.265259Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:16:55.113872Z",
     "start_time": "2017-10-26T00:16:51.149180Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "(2622081,) [364606, 113692, 197002, 330024, 326252, 151042, 75648, 1818, 276247, 61043, 228115, 326252, 74974, 301275, 76659, 326252, 361104, 329709, 253643, 205596, 153283, 326252, 80594, 326252, 113692, 18820, 349251, 59442, 123801, 228752, 245229, 307200, 17105, 60555, 69032, 1818, 274163, 151942, 246684, 222367, 253643, 243777, 274163, 50915, 274163, 12413, 1818, 228752, 364603, 232434, 214275, 235155, 163151, 123801, 101614, 101366, 364607]\n",
      "(2622081, 3) [364606, 97957, 364607]\n",
      "(2622081,) [364606, 326252, 364607]\n",
      "(2622081,) 6\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(np.array(x_train_21_T).shape, x_train_21_T[0])\n",
    "print(np.array(x_train_21_G).shape, x_train_21_G[0])\n",
    "print(np.array(x_train_21_V).shape, x_train_21_V[0])\n",
    "print(np.array(x_train_21_C).shape, x_train_21_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.385302Z",
     "start_time": "2017-10-26T00:16:55.114982Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.853790Z",
     "start_time": "2017-10-26T00:17:10.386619Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (293702,)\n",
      "gene (293702, 3) [364606, 112978, 364607]\n",
      "variation (293702,) [364606, 295010, 364607]\n",
      "classes (293702,) 2\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_21_T).shape)\n",
    "print(\"gene\",np.array(x_val_21_G).shape, x_val_21_G[0])\n",
    "print(\"variation\",np.array(x_val_21_V).shape, x_val_21_V[0])\n",
    "print(\"classes\",np.array(x_val_21_C).shape, x_val_21_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.857143Z",
     "start_time": "2017-10-26T00:17:10.854958Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.872469Z",
     "start_time": "2017-10-26T00:17:10.858162Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_SENT_LEN = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.327704Z",
     "start_time": "2017-10-26T00:17:10.873609Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2622081, 60) (293702, 60)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_21_T.shape, x_val_21_T.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.564842Z",
     "start_time": "2017-10-26T00:17:27.328915Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_21_C = np.array(x_train_21_C) - 1\n",
    "x_val_21_C = np.array(x_val_21_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.631996Z",
     "start_time": "2017-10-26T00:17:27.566066Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2622081, 9) (293702, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)\n",
    "x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)\n",
    "print(x_train_21_C.shape, x_val_21_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:text_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:43.191953Z",
     "start_time": "2017-10-26T00:32:43.173899Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"text\",\n",
    "         \"divide_document\": \"single_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:53.910517Z",
     "start_time": "2017-10-26T00:32:43.193444Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:54.978928Z",
     "start_time": "2017-10-26T00:32:53.911691Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (2988,)\n",
      "gene (2988, 3) [352216, 164788, 352217]\n",
      "variation (2988,) [352216, 86196, 352217]\n",
      "classes (2988,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_22_T).shape)\n",
    "print(\"gene\",np.array(x_train_22_G).shape, x_train_22_G[0])\n",
    "print(\"variation\",np.array(x_train_22_V).shape, x_train_22_V[0])\n",
    "print(\"classes\",np.array(x_train_22_C).shape, x_train_22_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.258929Z",
     "start_time": "2017-10-26T00:32:54.980078Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:24:03.444204Z",
     "start_time": "2017-10-26T21:24:03.330601Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_22_T).shape)\n",
    "print(\"gene\",np.array(x_val_22_G).shape, x_val_22_G[0])\n",
    "print(\"variation\",np.array(x_val_22_V).shape, x_val_22_V[0])\n",
    "print(\"classes\",np.array(x_val_22_C).shape, x_val_22_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.437499Z",
     "start_time": "2017-10-26T00:32:56.413156Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.450804Z",
     "start_time": "2017-10-26T00:32:56.439121Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_TEXT_LEN = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.501666Z",
     "start_time": "2017-10-26T00:32:56.452013Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5000) (333, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_22_T.shape, x_val_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.536590Z",
     "start_time": "2017-10-26T00:32:57.502761Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 1) (2988, 4)\n",
      "(333, 1) (333, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_train_22_G.shape, x_train_22_V.shape)\n",
    "print(x_val_22_G.shape, x_val_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.542037Z",
     "start_time": "2017-10-26T00:32:57.538132Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_22_C = np.array(x_train_22_C) - 1\n",
    "x_val_22_C = np.array(x_val_22_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.576726Z",
     "start_time": "2017-10-26T00:32:57.543764Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 9) (333, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)\n",
    "x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)\n",
    "print(x_train_22_C.shape, x_val_22_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### test Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:32.887420Z",
     "start_time": "2017-09-26T03:43:29.372697Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)\n",
    "x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                has_class=False,\n",
    "                                                                add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.178763Z",
     "start_time": "2017-09-26T03:43:32.888877Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test data\n",
      "text (986,)\n",
      "gene (986, 3) [364606, 188717, 364607]\n",
      "variation (986,) [364606, 317947, 364607]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test data\")\n",
    "print(\"text\",np.array(x_test_22_T).shape)\n",
    "print(\"gene\",np.array(x_test_22_G).shape, x_test_22_G[0])\n",
    "print(\"variation\",np.array(x_test_22_V).shape, x_test_22_V[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.546461Z",
     "start_time": "2017-09-26T03:43:33.181689Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_test_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.575305Z",
     "start_time": "2017-09-26T03:43:33.548386Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 1) (986, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_test_22_G.shape, x_test_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T:text_chars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:07:49.588270Z",
     "start_time": "2017-10-26T23:07:49.582681Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"          : \"raw_chars\",\n",
    "         \"variation_unit\"     : \"raw_chars\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"           : \"raw_chars\",\n",
    "         \"doc_form\"           : \"text\",\n",
    "         \"divide_document\"    : \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:09:42.095904Z",
     "start_time": "2017-10-26T23:07:50.005896Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_33_T, x_train_33_G, x_train_33_V, x_train_33_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:09:48.474776Z",
     "start_time": "2017-10-26T23:09:42.097144Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (1086419,) [74, 71, 19, 7, 4, 72, 71, 19, 20, 12, 14, 17, 72, 71, 18, 20, 15, 15, 17, 4, 18, 18, 14, 17, 72, 71, 6, 4, 13, 4, 72, 71, 15, 19, 4, 13, 72, 71, 8, 18, 72, 71, 5, 17, 4, 16, 20, 4, 13, 19, 11, 24, 72, 71, 12, 20, 19, 0, 19, 4, 3, 72, 71, 8, 13, 72, 71, 3, 8, 21, 4, 17, 18, 4, 72, 71, 7, 20, 12, 0, 13, 72, 71, 2, 0, 13, 2, 4, 17, 18, 72, 71, 0, 13, 3, 72, 71, 8, 13, 72, 71, 0, 20, 19, 14, 18, 14, 12, 0, 11, 72, 71, 3, 14, 12, 8, 13, 0, 13, 19, 72, 71, 2, 0, 13, 2, 4, 17, 72, 71, 15, 17, 4, 3, 8, 18, 15, 14, 18, 8, 19, 8, 14, 13, 72, 71, 3, 8, 18, 14, 17, 3, 4, 17, 18, 72, 71, 72, 75]\n",
      "gene (1086419,) [74, 71, 15, 19, 4, 13, 72, 75]\n",
      "variation (1086419,) [74, 71, 24, 27, 32, 2, 72, 75]\n",
      "classes (1086419,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_33_T).shape, x_train_33_T[0])\n",
    "print(\"gene\",np.array(x_train_33_G).shape, x_train_33_G[0])\n",
    "print(\"variation\",np.array(x_train_33_V).shape, x_train_33_V[0])\n",
    "print(\"classes\",np.array(x_train_33_C).shape, x_train_33_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:05.745670Z",
     "start_time": "2017-10-26T23:09:48.475925Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_33_T, x_val_33_G, x_val_33_V, x_val_33_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:06.510202Z",
     "start_time": "2017-10-26T23:10:05.746898Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (128341,) [74, 71, 0, 19, 72, 71, 19, 7, 8, 18, 72, 71, 19, 8, 12, 4, 72, 71, 15, 14, 8, 13, 19, 72, 71, 72, 71, 19, 7, 4, 72, 71, 4, 23, 15, 17, 4, 18, 18, 8, 14, 13, 72, 71, 14, 5, 72, 71, 22, 8, 11, 3, 36, 19, 24, 15, 4, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 72, 71, 20, 28, 14, 18, 72, 71, 2, 4, 11, 11, 18, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 15, 14, 19, 4, 13, 19, 72, 71, 2, 4, 11, 11, 72, 71, 2, 24, 2, 11, 4, 72, 71, 0, 17, 17, 4, 18, 19, 72, 71, 0, 19, 72, 71, 1, 14, 19, 7, 72, 71, 19, 4, 12, 15, 4, 17, 0, 19, 20, 17, 4, 18, 72, 71, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 18, 36, 15, 7, 0, 18, 4, 72, 71, 8, 13, 7, 8, 1, 8, 19, 8, 14, 13, 72, 71, 14, 5, 72, 71, 30, 28, 33, 29, 72, 71, 72, 71, 0, 13, 3, 72, 71, 30, 35, 33, 29, 72, 71, 72, 71, 0, 19, 72, 71, 29, 33, 27, 2, 72, 71, 0, 13, 3, 72, 71, 30, 26, 27, 2, 72, 71, 72, 71, 17, 4, 18, 15, 4, 2, 19, 8, 21, 4, 11, 24, 72, 71, 72, 75]\n",
      "gene (128341,) [74, 71, 2, 3, 10, 13, 28, 0, 72, 75]\n",
      "variation (128341,) [74, 71, 0, 32, 26, 21, 72, 75]\n",
      "classes (128341,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_33_T).shape, x_val_33_T[98])\n",
    "print(\"gene\",np.array(x_val_33_G).shape, x_val_33_G[0])\n",
    "print(\"variation\",np.array(x_val_33_V).shape, x_val_33_V[0])\n",
    "print(\"classes\",np.array(x_val_33_C).shape, x_val_33_C[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:06.513430Z",
     "start_time": "2017-10-26T23:10:06.511325Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:06.527659Z",
     "start_time": "2017-10-26T23:10:06.514422Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_CHAR_IN_SENT_LEN = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:18.903599Z",
     "start_time": "2017-10-26T23:10:06.529246Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 150) (128341, 150)\n"
     ]
    }
   ],
   "source": [
    "x_train_33_T = pad_sequences(x_train_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_33_T = pad_sequences(x_val_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_33_T.shape, x_val_33_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:27.876369Z",
     "start_time": "2017-10-26T23:10:18.905017Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 150) (1086419, 150)\n",
      "(128341, 150) (128341, 150)\n"
     ]
    }
   ],
   "source": [
    "x_train_33_G = pad_sequences(x_train_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "x_train_33_V = pad_sequences(x_train_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "\n",
    "x_val_33_G = pad_sequences(x_val_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "x_val_33_V = pad_sequences(x_val_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "\n",
    "print(x_train_33_G.shape, x_train_33_V.shape)\n",
    "print(x_val_33_G.shape, x_val_33_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:27.985411Z",
     "start_time": "2017-10-26T23:10:27.877544Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_33_C = np.array(x_train_33_C) - 1\n",
    "x_val_33_C = np.array(x_val_33_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:28.016098Z",
     "start_time": "2017-10-26T23:10:27.986732Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 9) (128341, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_33_C = np_utils.to_categorical(np.array(x_train_33_C), 9)\n",
    "x_val_33_C = np_utils.to_categorical(np.array(x_val_33_C), 9)\n",
    "print(x_train_33_C.shape, x_val_33_C.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### for words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.595494Z",
     "start_time": "2017-10-26T00:32:57.579320Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "WORD_EMB_SIZE = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:33:20.867764Z",
     "start_time": "2017-10-26T00:32:57.597200Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(352220, 200)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "ft_file_path = \"/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec\"\n",
    "trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)\n",
    "trained_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:28.019172Z",
     "start_time": "2017-10-26T23:10:28.017168Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CHAR_EMB_SIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:28.051353Z",
     "start_time": "2017-10-26T23:10:28.020527Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 64)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)\n",
    "char_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:12:00.322178Z",
     "start_time": "2017-10-26T23:12:00.259122Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import tensorflow.contrib.keras as keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.engine import Layer, InputSpec, InputLayer\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "\n",
    "from keras.layers import Dropout, Embedding, concatenate\n",
    "from keras.layers import Conv1D, MaxPooling1D, Conv2D, MaxPooling2D, ZeroPadding1D\n",
    "from keras.layers import Dense, Input, Flatten, BatchNormalization\n",
    "from keras.layers import Concatenate, Dot, Merge, Multiply, RepeatVector\n",
    "from keras.layers import Bidirectional, TimeDistributed\n",
    "from keras.layers import SimpleRNN, LSTM, GRU, Lambda, Permute\n",
    "\n",
    "from keras.layers.core import Reshape, Activation\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint,EarlyStopping,TensorBoard\n",
    "from keras.constraints import maxnorm\n",
    "from keras.regularizers import l2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_1: paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:12:02.903111Z",
     "start_time": "2017-10-26T23:12:02.180367Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_1 = Sequential([\n",
    "    Embedding(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE, weights=[char_embeddings], \n",
    "               input_length=MAX_CHAR_IN_SENT_LEN, trainable=True),\n",
    "    Conv1D(64, 3, padding=\"valid\")\n",
    "])\n",
    "\n",
    "# 4 pairs of convolution blocks followed by pooling\n",
    "for filter_size in [64, 128, 256, 512]:\n",
    "    \n",
    "    # each iteration is a convolution block\n",
    "    for cb_i in [0,1]:\n",
    "        model_1.add(Conv1D(filter_size, 3, padding=\"same\"))\n",
    "        model_1.add(BatchNormalization())\n",
    "        model_1.add(Activation(\"relu\"))\n",
    "        model_1.add(Conv1D(filter_size, 3, padding=\"same\")),\n",
    "        model_1.add(BatchNormalization())\n",
    "        model_1.add(Activation(\"relu\"))\n",
    "    \n",
    "    model_1.add(MaxPooling1D(pool_size=2, strides=3))\n",
    "\n",
    "# model.add(KMaxPooling(k=2))\n",
    "model_1.add(Flatten())\n",
    "model_1.add(Dense(4096, activation=\"relu\"))\n",
    "model_1.add(Dense(2048, activation=\"relu\"))\n",
    "model_1.add(Dense(2048, activation=\"relu\"))\n",
    "model_1.add(Dense(9, activation=\"softmax\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:12:04.497079Z",
     "start_time": "2017-10-26T23:12:04.410170Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 150, 64)           4800      \n",
      "_________________________________________________________________\n",
      "conv1d_6 (Conv1D)            (None, 148, 64)           12352     \n",
      "_________________________________________________________________\n",
      "conv1d_7 (Conv1D)            (None, 148, 64)           12352     \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 148, 64)           256       \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 148, 64)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_8 (Conv1D)            (None, 148, 64)           12352     \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 148, 64)           256       \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 148, 64)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_9 (Conv1D)            (None, 148, 64)           12352     \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 148, 64)           256       \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 148, 64)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_10 (Conv1D)           (None, 148, 64)           12352     \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 148, 64)           256       \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 148, 64)           0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 49, 64)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_11 (Conv1D)           (None, 49, 128)           24704     \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 49, 128)           512       \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 49, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 49, 128)           49280     \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 49, 128)           512       \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 49, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_13 (Conv1D)           (None, 49, 128)           49280     \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 49, 128)           512       \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 49, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, 49, 128)           49280     \n",
      "_________________________________________________________________\n",
      "batch_normalization_12 (Batc (None, 49, 128)           512       \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 49, 128)           0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 16, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, 16, 256)           98560     \n",
      "_________________________________________________________________\n",
      "batch_normalization_13 (Batc (None, 16, 256)           1024      \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 16, 256)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_16 (Conv1D)           (None, 16, 256)           196864    \n",
      "_________________________________________________________________\n",
      "batch_normalization_14 (Batc (None, 16, 256)           1024      \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 16, 256)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_17 (Conv1D)           (None, 16, 256)           196864    \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc (None, 16, 256)           1024      \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 16, 256)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_18 (Conv1D)           (None, 16, 256)           196864    \n",
      "_________________________________________________________________\n",
      "batch_normalization_16 (Batc (None, 16, 256)           1024      \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 16, 256)           0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 5, 256)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_19 (Conv1D)           (None, 5, 512)            393728    \n",
      "_________________________________________________________________\n",
      "batch_normalization_17 (Batc (None, 5, 512)            2048      \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 5, 512)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_20 (Conv1D)           (None, 5, 512)            786944    \n",
      "_________________________________________________________________\n",
      "batch_normalization_18 (Batc (None, 5, 512)            2048      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 5, 512)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_21 (Conv1D)           (None, 5, 512)            786944    \n",
      "_________________________________________________________________\n",
      "batch_normalization_19 (Batc (None, 5, 512)            2048      \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 5, 512)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_22 (Conv1D)           (None, 5, 512)            786944    \n",
      "_________________________________________________________________\n",
      "batch_normalization_20 (Batc (None, 5, 512)            2048      \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 5, 512)            0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 2, 512)            0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 4096)              4198400   \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2048)              8390656   \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 2048)              4196352   \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 9)                 18441     \n",
      "=================================================================\n",
      "Total params: 20,502,025\n",
      "Trainable params: 20,494,345\n",
      "Non-trainable params: 7,680\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])\n",
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:12:11.520633Z",
     "start_time": "2017-10-26T23:12:11.515665Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:12:12.305158Z",
     "start_time": "2017-10-26T23:12:12.299617Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"model_1_weights.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:12:12.743356Z",
     "start_time": "2017-10-26T23:12:12.740357Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "earlystopping = EarlyStopping(monitor='val_categorical_accuracy', \n",
    "                              min_delta=0, patience=5, \n",
    "                              verbose=1, mode='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:39:04.607259Z",
     "start_time": "2017-10-26T23:12:30.023614Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no checkpoints available !\n",
      "Train on 1086419 samples, validate on 128341 samples\n",
      "Epoch 1/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.5828 - categorical_accuracy: 0.4186Epoch 00000: val_categorical_accuracy improved from -inf to 0.46086, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 318s - loss: 1.5828 - categorical_accuracy: 0.4186 - val_loss: 1.4530 - val_categorical_accuracy: 0.4609\n",
      "Epoch 2/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.4081 - categorical_accuracy: 0.4743Epoch 00001: val_categorical_accuracy did not improve\n",
      "1086419/1086419 [==============================] - 317s - loss: 1.4081 - categorical_accuracy: 0.4743 - val_loss: 1.4503 - val_categorical_accuracy: 0.4583\n",
      "Epoch 3/5\n",
      "1086208/1086419 [============================>.] - ETA: 0s - loss: 1.3392 - categorical_accuracy: 0.4991Epoch 00002: val_categorical_accuracy improved from 0.46086 to 0.49781, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 317s - loss: 1.3392 - categorical_accuracy: 0.4991 - val_loss: 1.3618 - val_categorical_accuracy: 0.4978\n",
      "Epoch 4/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.2974 - categorical_accuracy: 0.5141Epoch 00003: val_categorical_accuracy did not improve\n",
      "1086419/1086419 [==============================] - 319s - loss: 1.2973 - categorical_accuracy: 0.5141 - val_loss: 1.3735 - val_categorical_accuracy: 0.4872\n",
      "Epoch 5/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.2667 - categorical_accuracy: 0.5260Epoch 00004: val_categorical_accuracy improved from 0.49781 to 0.49994, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 317s - loss: 1.2667 - categorical_accuracy: 0.5260 - val_loss: 1.3507 - val_categorical_accuracy: 0.4999\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_1.load_weights(\"model_1_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    model_1.fit(x_train_33_T, x_train_33_C, \n",
    "          validation_data=(x_val_33_T, x_val_33_C),\n",
    "          epochs=5, batch_size=128,shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T00:17:11.842299Z",
     "start_time": "2017-10-26T23:50:37.855766Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1086419 samples, validate on 128341 samples\n",
      "Epoch 1/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.2381 - categorical_accuracy: 0.5385Epoch 00000: val_categorical_accuracy improved from 0.49994 to 0.51484, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 319s - loss: 1.2381 - categorical_accuracy: 0.5385 - val_loss: 1.3362 - val_categorical_accuracy: 0.5148\n",
      "Epoch 2/5\n",
      "1086208/1086419 [============================>.] - ETA: 0s - loss: 1.2116 - categorical_accuracy: 0.5489Epoch 00001: val_categorical_accuracy improved from 0.51484 to 0.51622, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 317s - loss: 1.2116 - categorical_accuracy: 0.5489 - val_loss: 1.3443 - val_categorical_accuracy: 0.5162\n",
      "Epoch 3/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.1928 - categorical_accuracy: 0.5556Epoch 00002: val_categorical_accuracy improved from 0.51622 to 0.51717, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 317s - loss: 1.1928 - categorical_accuracy: 0.5556 - val_loss: 1.3582 - val_categorical_accuracy: 0.5172\n",
      "Epoch 4/5\n",
      "1086336/1086419 [============================>.] - ETA: 0s - loss: 1.1728 - categorical_accuracy: 0.5620Epoch 00003: val_categorical_accuracy did not improve\n",
      "1086419/1086419 [==============================] - 317s - loss: 1.1728 - categorical_accuracy: 0.5620 - val_loss: 1.3455 - val_categorical_accuracy: 0.5142\n",
      "Epoch 5/5\n",
      "1086208/1086419 [============================>.] - ETA: 0s - loss: 1.1578 - categorical_accuracy: 0.5675Epoch 00004: val_categorical_accuracy improved from 0.51717 to 0.52600, saving model to model_1_weights.hdf5\n",
      "1086419/1086419 [==============================] - 319s - loss: 1.1578 - categorical_accuracy: 0.5676 - val_loss: 1.3162 - val_categorical_accuracy: 0.5260\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_1.load_weights(\"model_1_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    model_1.fit(x_train_33_T, x_train_33_C, \n",
    "          validation_data=(x_val_33_T, x_val_33_C),\n",
    "          epochs=5, batch_size=128,shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "833px",
    "left": "0px",
    "right": "1192px",
    "top": "52px",
    "width": "300px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
